package crawler.web.engine.custom;

import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.regex.Pattern;

import javax.annotation.Resource;

import library.functions.Function2PR;

import org.apache.commons.validator.routines.UrlValidator;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;

import crawler.web.beans.CrawlURL;
import crawler.web.engine.config.SpBN;
import crawler.web.framework.Crawler;

/**
 * @author waynechen Feb 24, 2012 3:37:27 PM
 */
@Component(value = "URLFilter")
public class URLFilter implements Function2PR<Crawler, CrawlURL, Boolean> {

	@Resource(name = SpBN.conf_maxLayer)
	private Integer maxLayer;

	@Autowired(required = true)
	public URLFilter(@Value("#{T(crawler.web.engine.config.ConfigBatchInit).getStringList('urlRegex')}") LinkedList<String> list) {
		for (String regex : list) {
			if (regex.length() <= 1)
				continue;
			else if (regex.startsWith("-")) {
				blackList.add(Pattern.compile(regex.substring(1)));
			} else if (regex.startsWith("+")) {
				whiteList.add(Pattern.compile(regex.substring(1)));
			} else {
				whiteList.add(Pattern.compile(regex));
			}
		}
	}

	private UrlValidator urlValidator = new UrlValidator();
	private List<Pattern> whiteList = new ArrayList<Pattern>();
	private List<Pattern> blackList = new ArrayList<Pattern>();

	private boolean isWhite(String urlStr) {
		for (Pattern p : whiteList) {
			while (p.matcher(urlStr).find())
				return true;
		}
		return false;
	}

	private boolean isBlack(String urlStr) {
		for (Pattern p : blackList) {
			if (p.matcher(urlStr).find())
				return true;
		}
		return false;
	}

	@Override
	public Boolean apply(Crawler crawler, CrawlURL url) {
		int maxlayer = 1;
		String urlStr = url.getOriUrl().trim();

		//crawler为空说明是初始化url
		if (crawler != null)
			maxlayer = this.maxLayer;

		//达到最大深度, 丢弃此url
		if (url.getLayer() >= maxlayer)
			return false;

		//对url进行正则过滤
		if (!isWhite(urlStr))
			return false;
		if (isBlack(urlStr))
			return false;

		//检查此url, 是否有效(从形式上)
		if (!urlValidator.isValid(urlStr))
			return false;

		modifyURL(url);
		//默认返回true
		return true;
	}

	/**
	 * 稍微修改url 如, http://xxx#!
	 * 
	 * @author waynechen
	 * @param url
	 */
	private void modifyURL(CrawlURL url) {
		String urlStr = url.getOriUrl().trim();
		if (urlStr.endsWith("#!")) {
			urlStr = urlStr.substring(0, urlStr.length() - 2);
			url.setOriUrl(urlStr);
		}
	}
}

//
//
//
//
//
//
//
//
//
//
//